Pi-Home Voice

Our project showed that many smart connected home devices could be easily be integrated into our voice-to-command framework. For future work, we look to explore adding more commands to our model for controlling HVAC, electric window shades, smart dimmers, color changing light bulbs, and thermostat devices. We also hope to explore more consistent connection types such as Bluetooth if the devices expect to be in close range, or creating a WIFI server that never changes address for the client to always be able to find. We would also like to improve the security of our device by improving the accuracy of our facial recognition and adding other layers of protection to our device (maybe a numerical passcode). Finally, we may add multiple users to the device and set up profiles for each and what devices they may be able to control. This means an updated GUI which has settings that may be configured and profiles that may be learned for a given user of our device. With this we can look to implement scheduled commands and automations (turn the lights off every day after 11 PM).

main.py

                
                    
                    import time
                    from time import sleep
                    import pygame, pigame
                    from pygame.locals import *
                    from threading import Thread
                    import sys
                    from pathlib import Path
                    import RPi.GPIO as GPIO
                    import subprocess 
                    import speech_recognition as sr
                    from speech_recognition.recognizers import google, whisper
                    from joblib import load
                    import numpy as np
                    import socket
                    import time
                    import os 
                    import io
                    import picamera
                    import face_recognition
                    
                    #uncomment to display on the PiTFT
                    #os.putenv('SDL_VIDEODRV','fbcon')
                    #os.putenv('SDL_FBDEV','/dev/fb0')
                    #os.putenv('SDL_MOUSEDRV','dummy')
                    #os.putenv('SDL_MOUSEDEV','/dev/null')
                    #os.putenv('DISPLAY','')
                    
                    #thread to configure the facial ID
                    def task_3():
                    	global user_img
                    	global result_list
                    	global secure_login	
                    	global camera
                    	result_list = []
                    	camera.capture('/home/pi/new/project/unknown.jpg') #take a picture using Pi Camera
                    	unknown_image = face_recognition.load_image_file('/home/pi/new/project/unknown.jpg')
                    	try:
                    		unknown_encoding = face_recognition.face_encodings(unknown_image)[0]
                    		for encodings in user_img:
                    			if len(encodings) > 0:
                    				result = face_recognition.compare_faces([encodings], unknown_encoding)
                    				result_list.append(result)
                    			else:
                    				result_list.append([False])
                    				
                    		flattened_list = [item for sublist in result_list for item in sublist]
                    		count_true = sum(flattened_list)
                    		count_false = len(flattened_list) - count_true
                    		print('true: ', count_true)
                    		print('false: ', count_false)
                    		if count_true >= 8: #compare camera capture with usr_img profile. If greater than 8, log in
                    			secure_login = 1
                    	except:
                    		pass
                    		
                    	while(1):
                    		break
                    
                    
                    
                    
                    #WIFI server thread
                    def task_2():	
                    	global wifi
                    	global connection 
                    	run = 1
                    	while(run): #Test if PI-4 is connected to a WIFI network before proceeding
                    		try:
                    			#socket.setdefaulttimeout(3)
                    			s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                    			s.connect(("8.8.8.8",53))
                    			s.close()
                    			run = 0
                    			wifi = 1 #Pi-4 is connected to a wifi network
                    		except:	
                    			run = 1
                    	def start_server(): #now start the server
                    		global new_predictions
                    		global connection
                    		IP = subprocess.run(['hostname', '-I'], check=True, stdout=subprocess.PIPE, text=True)
                    		#convert input to str
                    		host = str(IP.stdout.strip()) #start connection using IP of current WIFI network connection
                    		port = 6000    # Port number
                    
                    
                    		# Create a socket object
                    		with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as server_socket:
                    			# Bind to the host and port
                    			server_socket.bind((host, port))
                    
                    			# Start listening for connectios
                    			server_socket.listen(1)
                    			#print(f"Server is listening on {host}:{port}")
                    
                    			# Accept a connection
                    			client_socket, addr = server_socket.accept()
                    			with client_socket:
                    				print('Connected by', addr)
                    				connection = 1
                    				while True:
                    					# Prepare data to send
                    					client_socket.sendall(new_predictions.encode()) #send command every second
                    					time.sleep(1)
                    
                    	if __name__ == '__main__':
                    		start_server()
                    
                    
                    
                    
                    #audio recording thread
                    def task_1():
                    	global new_predictions
                    	global recording 
                    	global run_thread 
                    	global text_arr
                    	recording = 1
                    	run_thread = 1
                    	#start recording para: card number, device number, duration
                    	result = subprocess.run(['sudo', 'arecord', '-D', 'plughw:1,0', '-d', '3', '/home/pi/new/project/new.wav'])
                    	#convert input to str
                    	recognizer = sr.Recognizer()
                    	try:
                    		with sr.AudioFile('/home/pi/new/project/new.wav') as source:
                    			audio_data = recognizer.record(source)
                    			text = recognizer.recognize_google(audio_data) #use google API for speech-to-text
                    			text_arr = np.append(text_arr, text)
                    			new_inputs = [text]
                    			new_predictions = model.predict(new_inputs) #convert the speech-to-text output to a command 1,2,...,7
                    			new_predictions = new_predictions[0]
                    			print(new_predictions)
                    	except:
                    		pass
                    
                    	#break the thread after running
                    	run_thread = 0
                    	recording = 0
                    	while(1):
                    		break
                    #quit button callback
                    def GPIO17_callback(channel):
                    	global run
                    	run = 0
                    	quit()
                    
                    	
                    
                    	
                    
                    # GPIOS Config
                    GPIO.setmode(GPIO.BCM)
                    GPIO.setup(17, GPIO.IN, pull_up_down=GPIO.PUD_UP)
                    GPIO.add_event_detect(17, GPIO.FALLING, callback = GPIO17_callback, bouncetime = 300)
                    #load our ML Random Forest model for text to command conversion
                    model = load('/home/pi/new/project/speech.joblib')
                    
                    
                    size = width, height = 320, 240
                    pygame.init()
                    lcd = pygame.display.set_mode((320,240))
                    lcd.fill((0,0,0))
                    pygame.display.update()
                    mic_icon = pygame.image.load('/home/pi/new/project/mic_icon.png')
                    pitft = pigame.PiTft()
                    # flag for code run
                    run = 1
                    recording_done = 0
                    new_predictions = ''
                    recording = 0
                    run_thread = 0
                    sid_id =''
                    send = False
                    text_arr = np.array([''])
                    wifi = 0
                    connection = 0
                    #start server thread
                    t2 = Thread(target=task_2)
                    t2.start()
                    # arr entries -> kitchen, living room, doorlock
                    indication_arr = np.array([0,0,0])
                    
                    font = pygame.font.SysFont('Arial', 15)
                    text_l = font.render('L Light', True, (255, 255, 255))
                    text_l_rect = text_l.get_rect()
                    text_l_rect.center = (40, 170)
                    
                    text_k = font.render('K Light', True, (255, 255, 255))
                    text_k_rect = text_k.get_rect()
                    text_k_rect.center = (100, 170)
                    
                    text_d = font.render('Door', True, (255, 255, 255))
                    text_d_rect = text_d.get_rect()
                    text_d_rect.center = (160, 170)
                    
                    text_lo = font.render('logged out', True, (255, 255, 255))
                    text_lo_rect = text_d.get_rect()
                    text_lo_rect.center = (237, 105)
                    
                    text_li = font.render('logged in', True, (255, 255, 255))
                    text_li_rect = text_d.get_rect()
                    text_li_rect.center = (237, 105)
                    
                    font = pygame.font.SysFont('Arial', 12)
                    text_login = font.render('Tap to log in/out -->', True, (255, 255, 255))
                    text_login_rect = text_d.get_rect()
                    text_login_rect.center = (200, 225)
                    
                    text_wifi_off = font.render('WIFI', True, (255, 0, 0))
                    text_wifi_off_rect = text_d.get_rect()
                    text_wifi_off_rect.center = (180, 30)
                    
                    text_wifi_on = font.render('WIFI', True, (0, 255, 0))
                    text_wifi_on_rect = text_d.get_rect()
                    text_wifi_on_rect.center = (180, 30)
                    
                    text_client_off = font.render('CLIENT', True, (255, 0, 0))
                    text_client_off_rect = text_d.get_rect()
                    text_client_off_rect.center = (167, 45)
                    
                    text_client_on = font.render('CLIENT', True, (0, 255, 0))
                    text_client_on_rect = text_d.get_rect()
                    text_client_on_rect.center = (167, 45)
                    
                    
                    #start Pi camera
                    camera = picamera.PiCamera()
                    camera.resolution = (128, 96)
                    rgb = bytearray(camera.resolution[0] * camera.resolution[1] *3)
                    
                    #for face recognition load the user profile
                    image_path = ['/home/pi/new/project/data/image_' + str(i) + '.jpg' for i in range(10)]
                    user_img = []
                    result_list = []
                    for path in image_path:
                    	image = face_recognition.load_image_file(path)
                    	encoding = face_recognition.face_encodings(image)[0]
                    	user_img.append(encoding)
                    ##################
                    secure_login = 0
                    
                    
                    while(run):
                    	if new_predictions == '1':
                    		indication_arr[0] = 1
                    	if new_predictions == '2':
                    		indication_arr[0] = 0
                    	if new_predictions == '3':
                    		indication_arr[1] = 1
                    	if new_predictions == '4': 
                    		indication_arr[1] = 0
                    	if new_predictions == '5':
                    		indication_arr[2] = 1
                    	if new_predictions == '6':
                    		indication_arr[2] = 0
                    	lcd.fill((0,0,0))
                    	#record button
                    	lcd.blit(pygame.transform.scale(mic_icon, (50, 50)), (20,10))
                    	#kitchen light
                    	pygame.draw.circle(lcd, (250,0,0), (40, 200), 10)
                    	#living room light
                    	pygame.draw.circle(lcd, (250,0,0), (100, 200), 10)
                    	#door lock
                    	pygame.draw.circle(lcd, (250,0,0), (160, 200), 10)
                    	#### facial recognition
                    	if secure_login == 0:
                    		pygame.draw.rect(lcd, (0,0,255), pygame.Rect(290,210,30,30))
                    	else:
                    		pygame.draw.rect(lcd, (0,255,0), pygame.Rect(290,210,30,30))
                    	
                    	#for i in range(text_arr.shape[0]):
                    	lcd.blit(text_l, text_l_rect)
                    	lcd.blit(text_k, text_k_rect)
                    	lcd.blit(text_d, text_d_rect)
                    	lcd.blit(text_login, text_login_rect)
                    	stream = io.BytesIO()
                    	camera.capture(stream, use_video_port=True, format = 'rgb')
                    	stream.seek(0)
                    	stream.readinto(rgb)
                    	stream.close()
                    	cam_img = pygame.image.frombuffer(rgb[0:(camera.resolution[0] * camera.resolution[1] * 3)], camera.resolution, 'RGB')
                    	lcd.blit(cam_img, (200,0))	
                    	for event in pygame.event.get():
                    		if(event.type == pygame.MOUSEBUTTONDOWN):
                    			x,y = pygame.mouse.get_pos()
                    		elif(event.type == pygame.MOUSEBUTTONUP):
                    			x,y = pygame.mouse.get_pos()
                    			#if click record button
                    			if 70 >= x >= 20 and 60 >= y >= 10 and run_thread == 0 and secure_login == 1:
                    				t1 = Thread(target=task_1)
                    				t1.start()
                    			if 320 >= x >= 290 and 240 >= y >= 210:
                    				if secure_login == 0:
                    					t3 = Thread(target=task_3)
                    					t3.start()
                    				else:
                    					secure_login = 0
                    				
                    	for i in range(len(indication_arr)):
                    		if (indication_arr[i]):
                    			pygame.draw.circle(lcd, (0,255,0), (40 + i*60, 200), 10)
                    			
                    	if recording == 1:
                    		pygame.draw.rect(lcd, (255,0,0), pygame.Rect(0,0,70,70))
                    	if secure_login == 0:
                    		lcd.blit(text_lo, text_lo_rect)
                    		pygame.draw.circle(lcd, (250,0,0), (257,129), 10)
                    	if secure_login == 1:
                    		lcd.blit(text_li, text_li_rect)
                    		pygame.draw.circle(lcd, (0,255,0), (257,129), 10)
                    	if wifi == 0:
                    		lcd.blit(text_wifi_off, text_wifi_off_rect)
                    	else:
                    		lcd.blit(text_wifi_on, text_wifi_on_rect)
                    	if connection == 0:
                    		lcd.blit(text_client_off, text_client_off_rect)
                    	else:
                    		lcd.blit(text_client_on, text_client_on_rect)
                    		
                    	pygame.display.update()
                    	pitft.update()
                    	sleep(0.01)
                    		
                    GPIO.cleanup()
                    quit()
                    
                

client.py

                
                    
                    import socket
                    import RPi.GPIO as GPIO
                    from time import sleep
                    
                    #setup GPIOs
                    GPIO.setmode(GPIO.BCM)
                    GPIO.setup(14, GPIO.OUT)
                    GPIO.setup(17,GPIO.OUT)
                    GPIO.output(17, GPIO.HIGH)
                    GPIO.output(17, GPIO.LOW)
                    GPIO.setup(12,GPIO.OUT)
                    GPIO.setup(20,GPIO.OUT)
                    GPIO.setup(21,GPIO.OUT)
                    pwm_0 = GPIO.PWM(12, 50) #50 frequency 
                    GPIO.output(20, GPIO.LOW)
                    GPIO.output(21, GPIO.LOW)
                    pwm_0.start(20)
                    temp = '99999'
                    
                    
                    def GPIO6_callback(channel):		
                    	GPIO.cleanup()
                    	quit()
                    GPIO.setup(6, GPIO.IN, pull_up_down=GPIO.PUD_UP)
                    GPIO.add_event_detect(6, GPIO.FALLING, callback = GPIO6_callback, bouncetime = 300)
                    def connect_to_server():
                    	run = 1
                    	while(run):
                    		try:
                    			s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                    			s.connect(("8.8.8.8",53))
                    			s.close()
                    			run = 0
                    		except:
                    			run = 1
                    	global temp
                    	
                    	host = '10.49.67.157'
                    	port = 6000
                    	
                    	#Create a socket object
                    	with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as client_socket:
                    		print("Connect to the server")
                    		client_socket.connect((host, port))
                    		print(f"Connected to {host}:{port}")
                    		
                    		while True:
                    			#Receive the response
                    			data = client_socket.recv(1024)
                    			if not data:
                    				sleep(1)
                    			print('Received:', data.decode())
                    			if data.decode() == '1' and temp != data.decode(): #turn on living room light
                    				temp = data.decode()
                    				GPIO.output(14, GPIO.HIGH)
                    			if data.decode() == '2' and temp != data.decode(): #turn off living room light
                    				temp = data.decode()
                    				GPIO.output(14, GPIO.LOW)
                    			if data.decode() == '3' and temp != data.decode(): #turn on kitchen light
                    				temp = data.decode()
                    				GPIO.output(17, GPIO.HIGH)
                    			if data.decode() == '4' and temp != data.decode(): #turn off kitchen light
                    				temp = data.decode()
                    				GPIO.output(17, GPIO.LOW)
                    			if data.decode() == '5' and temp != data.decode(): #open the door
                    				temp = data.decode()
                    				GPIO.output(20, GPIO.LOW)
                    				GPIO.output(21, GPIO.HIGH)
                    				sleep(1.3)
                    			if data.decode() == '6' and temp != data.decode(): #close the door
                    				temp = data.decode()
                    				GPIO.output(20, GPIO.HIGH)
                    				GPIO.output(21, GPIO.LOW)
                    				sleep(1.3)
                    			GPIO.output(20, GPIO.LOW)
                    			GPIO.output(21, GPIO.LOW)
                    			sleep(1)
                    				
                    if __name__ == '__main__':
                    	connect_to_server()
                    
                

test_ml.py

                
                    
                    import numpy as np
                    from sklearn.model_selection import train_test_split
                    from sklearn.feature_extraction.text import TfidfVectorizer
                    from sklearn.svm import SVC
                    from sklearn.pipeline import make_pipeline
                    from sklearn.metrics import classification_report
                    from joblib import dump
                    from sklearn.ensemble import RandomForestClassifier
                    
                    import pandas as pd
                    data = pd.read_csv("Book1.csv", usecols=['X_train '])
                    data = np.array([data])
                    X_train = np.array([''])
                    for i in range(130):
                        X_train = np.append(X_train, data[0][i][0])
                    X = np.delete(X_train, 0)
                    
                    data = pd.read_csv("Book1.csv", usecols=['Y_train '])
                    data = np.array([data])
                    Y_train = np.array([''])
                    for i in range(130):
                        Y_train = np.append(Y_train, data[0][i][0])
                    y = np.delete(Y_train, 0)
                    
                    
                    
                    
                    
                    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=42)
                    
                    model = make_pipeline(TfidfVectorizer(), RandomForestClassifier())
                    
                    
                    model.fit(X_train, y_train)
                    dump(model, 'speech.joblib')
                    
                    predictions = model.predict(np.array(['oino oi']))
                    predictions

                    
                

Book1.csv

                
                    
                X_train	Y_train
turn on light in living room	1
trun off light in living room	2
turn on light in kitchen	3
trun off light in kitchen	4
open door	5
close door	6
turn on door	5
turn off door	6
open light	7
close light	7
living room kitchen	7
living room bathroom on	7
living room bathroom off	7
open living room	7
open bathroom	7
light on in living room	1
turn light on in living room	1
living room turn on  light	1
turn on living room light	1
could you please turn on light in living room	1
get some light in living room	1
get some light in living room	1
turn off living room light	2
could you turn off living room light	2
switch off living room light	2
kill living room light	2
turn off light in living room	2
could you please open door	5
open door	5
let us open door	5
pop door open	5
open front door	5
close door	6
let us close door	6
shut door	6
check weather	7
get a coffee	7
when is coffee shop open	7
turn on volumn	7
turn off volumn	7
tell me a joke	7
what to do tonight	7
have some fun	7
is it rain	7
iddo oidno doqm Doiqmo	7
living room light on	1
turn on living room	1
turn living room light on	1
light up living room	1
make living room brighter	1
add more light to living room	1
brighten living room	1
make it brighter in living room	1
lights on in living room	1
make living room light	1
lower light in living room	2
lower living room light	2
turn off living room	2
turn light off in living room	2
dark in living room	2
stop here living room lights off	2
turn living room light off	2
turn lights down in living room	2
lower lighting in living room	2
dim living room	2
lower light level in living room	2
darken living room	2
lights out in living room	2
kitchen light on	3
turn on kitchen	3
turn kitchen light on	3
light up kitchen	3
make kitchen brighter	3
add more light to kitchen	3
brighten kitchen	3
make it brighter in kitchen	3
lights on in kitchen	3
make kitchen light	3
lower light in kitchen	4
lower kitchen light	4
turn off kitchen	4
turn light off in kitchen	4
dark in kitchen	4
stop here kitchen lights off	4
turn kitchen light off	4
turn lights down in kitchen	4
lower lighting in kitchen	4
dim kitchen	4
lower light level in kitchen	4
darken kitchen	4
lights out in kitchen	4
on light kitchen	3
kitchen on light	3
kitchen on light	3
kitchen off light	4
kitchen light off	4
light kitchen off	4
open door	5
make door open	5
crack door open	5
release door	5
swing door open	5
open door wide	5
slide open door	5
door open	5
Open Door	5
close door	6
door closed	6
close door	6
shut door	6
make door shut	6
close door	6
door shut	6
shut down	6
lock door	6
lock door	6
living kitchen	7
shut living room	7
open living room	7
close light	7
hello	7
how are you	7
hi	7
stop	7
try again	7
is this on	7
recording	7
light door	7
turn off light	7
turn on light	7
close light	7
open light	7
shut light	7
light on	7
light off	7